import os, sys, string
from datetime import datetime, time, date
from string import strip, replace

screen = []

#Get ALL htm and html files in the directory tree starting at directory d. 
#Output to directory o.
#Force = delete files in directory if set to True
def get_walk(d, o, force, image):
    global screen
    htmlFiles = []
    otherFiles = []
    directoryNames = []
    directoryNames.append(".\\"+d)
    if o == "":
        outputDir = os.getcwd()+"\\tidy-"+d+"\\"
    else:
        outputDir = o+"\\"
    
    print "Directory: \t"+os.getcwd()+"\\"+d+"\\"
    print "Output: \t" +outputDir+"\n"
    screen.append("Directory: \t"+os.getcwd()+"\\"+d+"\\")
    screen.append("Output: \t" +outputDir+"\n")
    
    for dirname, dirnames, filenames in os.walk(".\\"+d):
        for subdirname in dirnames:
            #print os.path.join(dirname, subdirname)
            directoryNames.append(os.path.join(dirname, subdirname))
        for filename in filenames:
            #print os.path.join(dirname, filename)
            
            if (str.lower(filename[-5:]) == ".html" or str.lower(filename[-4:]) == ".htm"):
                htmlFiles.append(os.path.join(dirname, filename))
            elif (str.lower(filename[-5:]) == ".jpeg" or str.lower(filename[-4:]) == ".jpg"):
                otherFiles.append(os.path.join(dirname, filename))
            elif (str.lower(filename[-5:]) == ".tiff"):
                otherFiles.append(os.path.join(dirname, filename))
            elif (str.lower(filename[-4:]) == ".gif"):
                otherFiles.append(os.path.join(dirname, filename))
            elif (str.lower(filename[-4:]) == ".bmp"):
                otherFiles.append(os.path.join(dirname, filename))
            elif (str.lower(filename[-4:]) == ".png"):
                otherFiles.append(os.path.join(dirname, filename))
            elif (str.lower(filename[-4:]) == ".pdf"):
                otherFiles.append(os.path.join(dirname, filename))
                
    newDirectoryNames = []
    #print "New Directory Names: "
    for dir in directoryNames:
        #print "\t"+dir
        if o == "":
            newDirectoryNames.append(".\\tidy-"+dir[2:])
        else:
            newDirectoryNames.append(".\\"+o+dir[2+len(d):])
    #print "\n"
    
    #print "Altered Directory Names:"
    for dir in newDirectoryNames:
        #print "\t"+dir
        if os.path.isdir(dir) == True:
            if force == True:
                for file in os.listdir(dir):
                    if file.find(".htm") != -1 or file.find(".html") != -1:
                        os.remove(dir+"\\"+file)
            else:
                print "Output directory already exists."
                exit()
    #print "\n"
    
    for dir in newDirectoryNames:
        try:
            if os.path.isdir(dir):
                os.removedirs(dir)
        except os.error:
            pass
        
    for dir in newDirectoryNames:
        #print "make dir: "+dir
        if not os.path.isdir(dir):
            os.mkdir(dir)
    
    #print "Other Files:"
    if image == True:
        print "Copying all non-html files:"
        screen.append("Copying all non-html files:")
        for file in otherFiles:
            newfile = ""
            if o == "":
                newfile = ".\\tidy-"+file[2:]
            else:
                newfile = ".\\"+o+file[2+len(d):]
            
            print "\t"+file,
            os.system("copy \""+file+"\" \""+newfile+"\"")
        print "\n"
        screen.append("\n")
    
    return htmlFiles

#Get ONLY the htm and html files in the directory d.
#Output to directory o.
#Force = delete files in directory if set to True. Will not delete directories.
def get_dir(d, o, force, image):
    global screen
    if o == "":
        outputDir = os.getcwd()+"\\tidy-"+d+"\\"
    else:
        outputDir = o+"\\"
        
    print "Directory: \t"+os.getcwd()+"\\"+d+"\\"
    print "Output: \t" +outputDir+"\n"
    screen.append("Directory: \t"+os.getcwd()+"\\"+d+"\\")
    screen.append("Output: \t" +outputDir+"\n")
    
    htmlFiles = []
    otherFiles = []
    for filename in os.listdir(os.getcwd()+"\\"+d+"\\"):
        #print "filename: "+filename
        if (filename[-5:] == ".html" or filename[-4:] == ".htm"):
            htmlFiles.append(".\\"+d+"\\"+filename)
        else:
            if(filename[2:].find(".") != -1):
                if (str.lower(filename[-5:]) == ".jpeg" or str.lower(filename[-4:]) == ".jpg"):
                    otherFiles.append(".\\"+d+"\\"+filename)
                elif (str.lower(filename[-5:]) == ".tiff"):
                    otherFiles.append(".\\"+d+"\\"+filename)
                elif (str.lower(filename[-4:]) == ".gif"):
                    otherFiles.append(".\\"+d+"\\"+filename)
                elif (str.lower(filename[-4:]) == ".bmp"):
                    otherFiles.append(".\\"+d+"\\"+filename)
                elif (str.lower(filename[-4:]) == ".png"):
                    otherFiles.append(".\\"+d+"\\"+filename)
                elif (str.lower(filename[-4:]) == ".pdf"):
                    otherFiles.append(".\\"+d+"\\"+filename)
                
    if os.path.isdir(outputDir):
        if force == True:
            for file in os.listdir(outputDir):
                if file.find(".htm") != -1 or file.find(".html") != -1:
                    os.remove(outputDir+file)
        else:
            print "Output directory already exists."
            exit()
    
    if not os.path.isdir(outputDir):
        os.mkdir(outputDir)
    
    #if not os.path.isdir(outputDir+d):
    #    os.mkdir(outputDir+d)
    
    #print "Other Files:"
    #print "image: "+str(image)
    if image == True:
        print "Copying all non-html files:"
        screen.append("Copying all non-html files:")
        for file in otherFiles:
            newfile = ""
            if o == "":
                newfile = ".\\tidy-"+file[2:]
            else:
                newfile = ".\\"+o+file[2+len(d):]
            
            print "\t"+file,
            os.system("copy \""+file+"\" \""+newfile+"\"")
        print "\n"
        screen.append("\n")
    
    return htmlFiles

#Generates the XHTML for the files given in htmlFiles and outputs them to directory o.
def gen_tidy_files(htmlFiles, o, options):
    global screen
    if o == "":
        o = "tidy-"
        temp = []
        for file in htmlFiles:
            temp.append(".\\"+o+file[2:])
    else:
        temp = []
        for file in htmlFiles:
            temp.append(".\\"+o+file[2+len(file.split("\\")[1]):])
        
    i = 0    
    while i < len(htmlFiles):
        otemp = ""
        filename = temp[i]
        print "\t"+strip(htmlFiles[i]),
        otemp = otemp + "\t"+strip(htmlFiles[i])
        j = 50 - len(htmlFiles[i])
        while j > 0:
            print "",
            otemp = otemp + " "
            j=j-1
        print strip(filename)
        otemp = otemp + strip(filename)
        
        if filename[-5:] == ".html":
            os.system("tidy " + options + " -o "+filename+" -f "+filename+"-errors.txt -asxhtml "+htmlFiles[i])
        else:
            os.system("tidy " + options + " -o "+filename+" -f "+filename+"-errors.txt -asxhtml "+htmlFiles[i])
        i = i+1
        screen.append(otemp)

#Prints given html Files in two columns on the screen
def print_files(htmlFiles):
    global screen
    if len(htmlFiles) > 150:
        print "Found "+str(len(htmlFiles))+" files for conversion."
        return
    
    print "Found files:"
    screen.append("Found files:")
    i=0
    j=0
    screen.append("\t")
    print "\t",
    for filename in htmlFiles:
        temp = ""
        print strip(filename),
        temp = temp + strip(filename)
        if i == 0:
            i= i+1
            j=50-len(filename)
            
            while j > 0:
                print "",
                temp = temp + " "
                j = j-1

        else:
            print "\n\t",
            temp = temp + "\n\t"
            i = 0
        screen.append(temp)
    print "\n"
    screen.append("\n")

#Goes through all the error files and adds all the errors found (along with the file it found them in) to a variable it passes back
def del_errors(htmlFiles, o):
    #if o == "":
    #    o = "tidy-"
    #    temp = []
    #    for file in htmlFiles:
    #        temp.append(".\\"+o+file[2:])
    #    newHtmlFiles = temp
    #else:
    #    temp = []
    #    for file in newHtmlFiles:
    #        temp.append(".\\"+o+file[2+len(file.split("\\")[1]):])
    #    newHtmlFiles = temp
    
    error = []
    print "\nCleaning Up Error Files..."
    for filename in htmlFiles:
        #print "\n\tFilename: "+filename
        if o == "":
            o = "tidy-"
            file = ".\\"+o+filename[2:]+"-errors.txt"
        else:
            file = ".\\"+o+filename[2+len(filename.split("\\")[1]):]+"-errors.txt"
        
        #print "\tAltered File: "+file
        content = ""
        if os.path.isfile(file):
            f = open(file, 'r')
            content = f.read()
            f.close()
        
            if content != '':
                #print "\tError found: "+file
                
                content = content.replace("<","")
                content = content.replace(">","")
                content = content.replace("\n", "<br />\n")
                
                error.append("<div class=\"error\"><a href=\"file://"+string.replace(os.getcwd()+"/"+filename,"\\","/")+"\">"+filename +"</a></div>\n<div class=\"errorText\">"+ content + "</div>\n\n")

            #print "\tNo Error: "+file
                #file = r""+os.path.splitext(file)[0] + ".txt"
            try:
                os.remove(file)
            except os.error:
                print "OS Error: "+file
    
    return error

#Writes out all errors found to a HTML file it creates. The HTML file is stored in logs
def write_errors(error, warning):
    now = datetime.today()
    errorCount = len(error)
    
    file = sys.argv[0]
    file = file.split("\\")
    file[-1] = "logs\\"
    filename = ""
    for token in file:
        filename = filename + token + "\\"
    filename = filename[:-2]
    
    if not os.path.isdir(filename):
        os.system("mkdir \""+filename+"\"")
    
    filename = filename+"\\errors_"+now.strftime("%y-%m-%d_%H-%M-%S")+".html"
    f = open(filename, 'w')
    
    f.write("<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\"\n")
    f.write("\t\"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">\n")
    f.write("<html xmlns=\"http://www.w3.org/1999/xhtml\">\n")
    f.write("<head>\n")
    f.write("<title>Tidy-Up Error Log - "+now.strftime("%y-%m-%d_%H-%M-%S")+".txt</title>\n")
    f.write("<style type=\"text/css\">\n")
    f.write(".error{ color:red; }\n")
    f.write(".errorText{ margin: 0px 0px 2em 2em; }\n")
    f.write(".directory{ font-weight:bold;text-size:1.5em; }\n")
    f.write(".time{ font-weight:bold;text-size:1.5em;margin-bottom:2em; }\n")
    f.write(".errorCount{ font-weight:bold;text-size:1em;margin-bottom:2em; }\n")
    f.write("</style>\n")
    f.write("</head>\n")
    f.write("<body>\n")
    f.write("<h1>Tidy-Up Error Log</h1>\n")
    f.write("<div class=\"directory\">Directory: "+os.getcwd()+"</div>\n")
    f.write("<div class=\"time\">Generated: "+ now.strftime("%B %d, %Y - %H:%M:%S %Z") +"</div>\n")
    
    if errorCount == 0:
        f.write("<div class=\"errorCount\">There were no ")
        if warning == True:
            f.write("warnings/")
        f.write("errors found.</div>")
    elif errorCount == 1:
        f.write("<div class=\"errorCount\">There was "+ str(errorCount) +" file found with ")
        if warning == True:
            f.write("warnings/")
        f.write("errors.</div>")
    else:
        f.write("<div class=\"errorCount\">There were "+ str(errorCount) +" files found with ")
        if warning == True:
            f.write("warnings/")
        f.write("errors.</div>")
    
    f.writelines(error)
    
    f.write("</body>\n")
    f.write("</html>")
    
    f.close()
    
    return filename
    
#Writes out all errors found to a HTML file it creates. The HTML file is stored in logs
def write_log(htmlFiles, d, o, error, warning, start):
    global screen
    now = datetime.today()
    errorCount = len(error)
    
    file = sys.argv[0]
    file = file.split("\\")
    file[-1] = "logs\\"
    filename = ""
    for token in file:
        filename = filename + token + "\\"
    filename = filename[:-2]
    
    if not os.path.isdir(filename):
        os.system("mkdir \""+filename+"\"")
    
    filename = filename+"\\log_"+now.strftime("%y-%m-%d_%H-%M-%S")+".html"
    f = open(filename, 'w')
    
    f.write("<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\"\n")
    f.write("\t\"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">\n")
    f.write("<html xmlns=\"http://www.w3.org/1999/xhtml\">\n")
    f.write("<head>\n")
    f.write("<title>Tidy-Up Log - "+now.strftime("%b %d, %y %H:%M:%S")+"</title>\n")
    f.write("<style type=\"text/css\">\n")
    f.write(".error{ color:red; }\n")
    f.write(".errorText{ margin: 0px 0px 2em 2em; }\n")
    f.write(".directory{ font-weight:bold;text-size:1.5em; }\n")
    f.write(".time{ font-weight:bold;text-size:1.5em;}\n")
    f.write(".errorCount{ font-weight:bold;text-size:1em;margin-bottom:2em; }\n")
    f.write(".file{ text-size:0.75em;}\n")
    f.write("</style>\n")
    f.write("</head>\n")
    f.write("<body>\n")
    f.write("<h1>Tidy-Up Log</h1>\n")
    f.write("<div class=\"directory\">Input Directory: "+os.getcwd()+"\\"+d+"</div>\n")
    f.write("<div class=\"directory\">Output Directory: "+os.getcwd()+"\\"+o+"</div>\n")
    f.write("<div class=\"time\">Started: "+ start.strftime("%B %d, %Y - %H:%M:%S %Z") +"</div>\n")
    f.write("<div class=\"time\">Ended: "+ now.strftime("%B %d, %Y - %H:%M:%S %Z") +"</div><br />\n")
    
    f.write("<div class=\"errorCount\">There were "+ str(len(htmlFiles)) +" files converted to xhtml.<br />")
    
    #f.write("<div class=\"directory\">Converted the following files:</div>\n")
    
    if errorCount == 0:
        f.write("There were no ")
        if warning == True:
            f.write("warnings/")
        f.write("errors found.")
    elif errorCount == 1:
        f.write("There was "+ str(errorCount) +" file found with ")
        if warning == True:
            f.write("warnings/")
        f.write("errors.")
    else:
        f.write("There were "+ str(errorCount) +" files found with ")
        if warning == True:
            f.write("warnings/")
        f.write("errors.")
    f.write("</div>\n")
    
    f.write("<h2>Errors:</h2>")
    f.writelines(error)
    
    f.write("<h2>Converted Files:</h2>")
    for file in htmlFiles:
        newfile = ""
        if o == "":
            newfile = ".\\tidy-"+file[2:]
        else:
            newfile = ".\\"+o+file[2+len(d):]
        f.write("<div class=\"file\">Old/New File: <a href=\"file://"+string.replace(os.getcwd()+"/"+file[2:],"\\","/")+"\">"+file[2:]+"</a> - <a href=\"file://"+string.replace(os.getcwd()+"/"+newfile[2:],"\\","/")+"\">"+newfile[2:]+"</a></div><br />\n")
    
    f.write("<h2>Program Output:</h2>")
    f.write("<pre>\n")
    for line in screen:
        f.write(line+"\n")
    #f.writelines(screen)
    f.write("\n</pre>")
    
    f.write("</body>\n")
    f.write("</html>")
    
    f.close()
    
    return filename    

print "\nTidy-Up"
print "===============\n"
    
#Handle Arguments to Script
warning = False
body = False
directory = ""
output = ""
options = ""
help = False
force = False
recursive = False
log = False
image = False
i=1
while i < len(sys.argv):
    if str.lower(sys.argv[i]) == "-warning":
        warning = True
        options = options+" --show-warnings yes"
    elif str.lower(sys.argv[i]) == "-body":
        body = True
        options = options+" --show-body-only yes"
    elif str.lower(sys.argv[i]) == "-d":
        directory = sys.argv[i+1]
        i = i+1
    elif str.lower(sys.argv[i]) == "-o":
        output = sys.argv[i+1]
        i = i+1
    elif str.lower(sys.argv[i]) == "-f":
        force = True
    elif str.lower(sys.argv[i]) == "-r":
        recursive = True
    elif str.lower(sys.argv[i]) == "-images" or str.lower(sys.argv[i]) == "-image":
        image = True
    elif str.lower(sys.argv[i]) == "-log":
        log = True
    elif str.lower(sys.argv[i]) == "-help" or str.lower(sys.argv[i]) == "--help":
        help = True
    elif str.lower(sys.argv[i]) == "\\?" or str.lower(sys.argv[i]) == "/?":
        help = True    
    i = i+1

#if user asked for help print it and exit
if help == True or len(sys.argv) == 1:
    print "Options:\n"
    print "-D <directory>: MANDATORY. This field specifies the location of the directory you wish to convert."
    print "-O <directory>: This field specifies the location of the directory you wish to output files to."
    print "\t\tIf not specified then a new directory will be used with 'tidy-' appended to the input directory."
    print "-Body: \t\tWill extract only the body of the documents."
    print "-F: \t\tForce output directory location. Will delete any files that exist in the specified location."
    print "-R: \t\tWill convert all children directories inside input directory as well."
    print "-Image: \tWill copy images from the source directory to the output directory."
    print "-Warning: \tLog all warnings. Warnings normally discarded."
    print "-Log: \t\tOutput a log after conversion has completed."
    print "\nExample:"
    print "\t"+sys.argv[0].split("\\")[-1]+" -D html -O xhtml -Warning -F -R"
    exit()

start = datetime.today()
    
#if no directory was specified exit. Mandatory field    
if directory == "":
    print "Error: -D <directory> is a mandatory field. Type "+sys.argv[0].split("\\")[-1]+" -help for help."
    exit()
else:
    if not os.path.isdir(".\\"+directory):
        print "Error: The directory specified does not exist."
        exit()

#load configuration. loads config.txt in the same directory as this python script
configFile = sys.argv[0].split("\\")
configFile[-1] = "config.txt"
config = ""
for token in configFile:
    config = config+"\\"+token
config = config[1:]
os.environ['HTML_TIDY']=config
print "Loaded Config File: "+ os.environ['HTML_TIDY']
screen.append("Loaded Config File: "+ os.environ['HTML_TIDY'])

#Get files to be converted. Either single directory or tree.
if recursive == False:
    htmlFiles = get_dir(directory, output, force, image)
else:
    htmlFiles = get_walk(directory, output, force, image)
print_files(htmlFiles) #print off the found files

#Generate Files
print "\nProcessing Files..."
screen.append("\nProcessing Files...")
gen_tidy_files(htmlFiles, output, options)

#Handle Log
error = del_errors(htmlFiles, output)
log_file = write_log(htmlFiles, directory, output, error, warning, start)
os.system("\""+log_file+"\"") #Open error log if any errors were found